diff --git a/src/_pytest/logging.py b/src/_pytest/logging.py
index 5dfd47887..22fbc56b5 100644
--- a/src/_pytest/logging.py
+++ b/src/_pytest/logging.py
@@ -437,9 +437,11 @@ class LogCaptureFixture:
         """
         logger_obj = logging.getLogger(logger)
         # Save the original log-level to restore it during teardown.
-        self._initial_logger_levels.setdefault(logger, logger_obj.level)
+        if logger not in self._initial_logger_levels:
+            self._initial_logger_levels[logger] = logger_obj.level
         logger_obj.setLevel(level)
-        self._initial_handler_level = self.handler.level
+        if self._initial_handler_level is None:
+            self._initial_handler_level = self.handler.level
         self.handler.setLevel(level)
 
     @contextmanager
